/* Copyright (C) 2011  Chris Mueller <ruunsmail@googlemail.com>
 *
 * This file is part of the Liquid Programming Language
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.

 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */



#include "scanner.h"
#include "context.h"
#include "array.h"
#include "map.h"
#include "structures.h"
#include <stdlib.h>
#include <string.h>
#include <assert.h>

#define NULL_TERMINATED -1
#define INVALID -1

#define LINE(S) (lq_array_size(S->file->lines))
#define WARNING(S) S->file->warnings
#define ERROR(S) S->file->errors
#define FILENAME(S) S->file->filename

struct Mapping {
	byte* string;
	enum LqTOK value;
};



static struct Mapping Keywords[] = {
	{"if", LQ_TOKEN_IF},
	{"then", LQ_TOKEN_THEN},
	{"else", LQ_TOKEN_ELSE},
	{"and", LQ_TOKEN_AND},
	{"or", LQ_TOKEN_OR},
	{"not", LQ_TOKEN_NOT},
        {"mod", LQ_TOKEN_MODULO},
	{"if", LQ_TOKEN_IF},
	{"match", LQ_TOKEN_MATCH},
	{"by", LQ_TOKEN_BY},
	{"fun", LQ_TOKEN_FUN},
        {"module", LQ_TOKEN_MODULE},
	{"let", LQ_TOKEN_LET}
};


static struct LqMap* KeywordMap = 0; 

enum NumberState {
	INT_START, 
	INT_BIN_BASE, 
	INT_HEX_BASE, 
	INT_OCT_BASE, 
	INT_BIN_WAIT, 
	INT_HEX_WAIT, 
	INT_OCT_WAIT, 
	INT_BIN_READ, 
	INT_HEX_READ, 
	INT_OCT_READ, 
	INT_DEC_READ
};


enum FloatpointState {
	FP_START, 
	FP_EXPONENT, 
	FP_DOT, 
	FP_DECIMAL, 
	FP_MINUSPLUS, 
	FP_FINAL
};


enum StringState {
	STR_EAT, 
	STR_ESCAPE, 
	STR_UNICODE, 
	STR_FINAL
};


enum IdentifierState {
	ID_ATOM,
	ID_IDENTIFIER,
	ID_EXTENDED
};


struct LqToken*
create_token(enum LqTOK value, const byte* text, size_t size, int line)
{
	struct LqToken* token = malloc(sizeof(struct LqToken));

	token->value = value;
	token->line = line;

	if(text != 0 && size > 0) {
		byte* string = calloc(size, sizeof(byte));
		strncpy(string, text, size);

		token->text = string;
	} else {
		token->text = 0;
	}

	return token;
}


void
lq_scanner_initialize(void)
{
	if(KeywordMap == 0) {
		KeywordMap = lq_map_new(lq_cast(lq_ordering_funptr, strcmp));

		size_t size = sizeof(Keywords) / sizeof(struct Mapping);
		size_t i;

		for(i = 0; i < size; i++)
			lq_map_insert(KeywordMap, Keywords[i].string, &Keywords[i].value);
	}
}



int
lq_scanner_line(struct LqContext* scanner)
{
        assert(scanner != 0);
        
        return scanner->current->line;
}


enum LqTOK
lq_scanner_token(struct LqContext* scanner)
{
        assert(scanner != 0);

	return scanner->current->value;
}

byte* 			
lq_scanner_string(struct LqContext* scanner)
{
        assert(scanner != 0);

        return scanner->current->text;
}



struct LqFile*
lq_scanner_file(struct LqContext* scanner)
{
	assert(scanner != 0);
	return scanner->file;
}




static struct LqToken*
read_floatpoint_number(struct LqContext* scanner, byte** stream)
{
	struct LqArray* buffer = scanner->buffer;
	byte* p = *stream;
	gunichar ch = g_utf8_get_char(p);

	enum FloatpointState state = FP_START;

	do {
		switch(state) {
			case FP_START:
				if(ch == 'e' || ch == 'E')
					state = FP_EXPONENT;
				else if(ch == '.')
					state = FP_DOT;
				else 
					assert(FALSE);
				break;

			case FP_EXPONENT:
				if(ch == '+' | ch == '-') {
					state = FP_MINUSPLUS;
				} else if('0' > ch || ch > '9') {
					lq_error_at(scanner->file, "Unexpected character found in floatpoint literal after exponent");
					lq_array_append(buffer, "0", 1);
					state = FP_FINAL;
					goto NO_APPEND_BUFFER;
				} else {
					state = FP_FINAL;
				}
				break;

			case FP_DOT:
				state = FP_DECIMAL;

				if('0' > ch || ch > '9') {
					lq_error_at(scanner->file, "Unexpected character found in floatpoint literal after dot");
					lq_array_append(buffer, "0", 1);
					goto NO_APPEND_BUFFER;
				}
				break;

			case FP_DECIMAL:
				if(ch == 'e') {
					state = FP_EXPONENT;
				} else if('0' > ch || ch > '9') {
					goto RETURN_TOKEN;
				}
				break;

			case FP_MINUSPLUS:
				state = FP_FINAL;

				if('0' > ch || ch > '9') {
					lq_error_at(scanner->file, "Unexpected character found in floatpoint literal after minus/plus");
					lq_array_append(buffer, "0", 1);
					goto NO_APPEND_BUFFER;
				}             
				break;

			case FP_FINAL:
				if('0' > ch || ch > '9')
					goto RETURN_TOKEN;
				break;
		}

		lq_array_append(buffer, p, 1);

NO_APPEND_BUFFER:
		p = g_utf8_next_char(p);
		ch = g_utf8_get_char(p);

	} while(ch != '\0');

RETURN_TOKEN:
	*stream = p;

	lq_array_append(buffer, "\0", 1);

	return create_token(LQ_TOKEN_FLOATPOINT, lq_array_pointer(buffer), lq_array_size(buffer), LINE(scanner));
}


static struct LqToken*
read_number(struct LqContext* scanner, byte** stream) 
{
	struct LqArray* buffer = scanner->buffer;
	byte* p = *stream;
	gunichar ch = g_utf8_get_char(p);

	enum LqTOK tok = LQ_TOKEN_EOF;
	enum NumberState state = INT_START;

	do {
		switch(state) {
			case INT_START:
				if(ch == '2')
					state = INT_BIN_BASE;
				else if(ch == '0')
					state = INT_HEX_BASE;
				else if(ch == '8')
					state = INT_OCT_BASE;
				else 
					state = INT_DEC_READ;
				break;

			case INT_BIN_BASE:
			case INT_HEX_BASE:
			case INT_OCT_BASE:
				if(ch == 'x') {
					if(state == INT_BIN_BASE)
						state = INT_BIN_WAIT;
					else if(state == INT_HEX_BASE)
						state = INT_HEX_WAIT;
					else if(state == INT_OCT_BASE)
						state = INT_OCT_WAIT;

					// remove base prefix in buffer
					lq_array_clear(buffer);
					goto NO_APPEND_BUFFER;
				}

				if('0' <= ch && ch <= '9') 
					state = INT_DEC_READ;
				else if(ch == '.' || ch == 'e' || ch == 'E') {
					struct LqToken* token = read_floatpoint_number(scanner, &p);
					*stream = p;
					return token;
				} else {
					tok = LQ_TOKEN_DECNUMBER;
					goto RETURN_TOKEN;
				}
				break;

			case INT_BIN_WAIT:
				if(ch != '0' && ch != '1') {
					lq_error_at(scanner->file, "Unexpected character found in binary integer literal");
					lq_array_append(buffer, "0", 1);
					goto NO_APPEND_BUFFER;
				}
				state = INT_BIN_READ;
				break;

			case INT_HEX_WAIT:
				if(('0' > ch || ch > '9') && ('A' > ch || ch > 'F')) {
					lq_error_at(scanner->file, "Unexpected character found in hexadecimal integer literal");
					lq_array_append(buffer, "0", 1);
					goto NO_APPEND_BUFFER;
				}
				state = INT_HEX_READ;
				break;

			case INT_OCT_WAIT:
				if('0' > ch || ch < '7') {
					lq_error_at(scanner->file, "Unexpected character found in octal integer literal");
					lq_array_append(buffer, "0", 1);
					goto NO_APPEND_BUFFER;
				}
				state = INT_OCT_READ;
				break;

			case INT_BIN_READ:
				tok = LQ_TOKEN_BINNUMBER;

				if(ch == '_')
					goto NO_APPEND_BUFFER;
				else if(ch != '0' && ch != '1') {
					goto RETURN_TOKEN;
				}
				break;


			case INT_OCT_READ:
				tok = LQ_TOKEN_OCTNUMBER;

				if(ch == '_')
					goto NO_APPEND_BUFFER;
				else if(('0' > ch || ch > '7')) {
					goto RETURN_TOKEN;
				}
				break;


			case INT_DEC_READ:
				tok = LQ_TOKEN_DECNUMBER;

				if(ch == '.' || ch == 'e' || ch == 'E') {
					struct LqToken* token = read_floatpoint_number(scanner, &p);
					*stream = p;
					return token;                    
				} else if(ch == '_')
					goto NO_APPEND_BUFFER;
				else if('0' > ch || ch > '9') {
					goto RETURN_TOKEN;
				}               
				break;


			case INT_HEX_READ:
				tok = LQ_TOKEN_HEXNUMBER;

				if(ch == '_')
					goto NO_APPEND_BUFFER;
				else if(('0' > ch || ch > '9') && ('A' > ch || ch > 'F')) {
					goto RETURN_TOKEN;
				}
				break;

			default:
				break;
		}

		lq_array_append(buffer, p, 1);

NO_APPEND_BUFFER:
		p = g_utf8_next_char(p);
		ch = g_utf8_get_char(p);

	} while(ch != '\0');

RETURN_TOKEN:
	*stream = p;

	lq_array_append(buffer, "\0", 1);

	return create_token(tok, lq_array_pointer(buffer), lq_array_size(buffer), LINE(scanner));
}


static struct LqToken*
read_identifier(struct LqContext* scanner, byte** stream)
{
	struct LqArray* buffer = scanner->buffer;
	byte* p = *stream;
	gunichar ch = g_utf8_get_char(p);

	enum LqTOK token = LQ_TOKEN_ATOM;
	gboolean has_extended = FALSE;

	while(g_unichar_isalnum(ch) || ch == '_' || ch == '?') {
		if (ch == '?')  
			has_extended = TRUE;

		switch(token) {
			case LQ_TOKEN_ATOM:
				if(g_unichar_islower(ch)) {
					token = has_extended 
						? LQ_TOKEN_EXTIDENTIFIER
						: LQ_TOKEN_IDENTIFIER;
				}
				break;
			case LQ_TOKEN_IDENTIFIER:
				if(ch == '?')
					token = LQ_TOKEN_EXTIDENTIFIER;
				break;
			case LQ_TOKEN_EXTIDENTIFIER:
				break;
			default:
				assert(0);
		}

		lq_array_append(buffer, p, g_unichar_to_utf8(ch, NULL));

		p = g_utf8_next_char(p);
		ch = g_utf8_get_char(p);
	}

	lq_array_append(buffer, "\0", 1);

	*stream = --p;

	enum LqTOK* tok = (token == LQ_TOKEN_IDENTIFIER) 
		? lq_map_lookup(KeywordMap, lq_array_pointer(buffer))
		: NULL;

	return (tok != NULL) 
		? create_token(*tok, 0, 0, LINE(scanner))
		: create_token(token, lq_array_pointer(buffer), lq_array_size(buffer), LINE(scanner));
}


static struct LqToken*
read_string(struct LqContext* scanner, byte** stream)
{
	struct LqArray* buffer = scanner->buffer;
	byte* p = g_utf8_next_char(*stream);
	enum StringState state = STR_EAT;
	gunichar ch = g_utf8_get_char(p);
	int unicount = 0;

	while(TRUE) {
		if(ch == '\0' && state != STR_FINAL) {
			lq_error_at(scanner->file, "Unexpected end of file found in unclosed string");

			while(unicount-- > 0)
				lq_array_append(buffer, "0", 1);

			if(state == STR_ESCAPE)
				lq_array_append(buffer, "0", 1);

			break;
		}

		switch(state) {
			case STR_EAT:
				if(ch == STRING_ESCAPE_SYMBOL) {
					state = STR_ESCAPE;
				} 
				else if(ch == '\"') {
					state = STR_FINAL;
					goto RETURN_TOKEN;
				}
				else if(ch == '\r' || ch == '\n') {
					lq_error_at(scanner->file, "Unexpected newline/carriage return found in this string literal");
					goto RETURN_TOKEN;
				}
				else if(g_unichar_isprint(ch) != TRUE) {
					lq_error_at(scanner->file, "Unexpected unprintable unicode character U+%X found in this string literal", ch);
					goto NO_BUFFER_APPEND;
				}
				break;

			case STR_ESCAPE:
				switch(ch) {
					case 'a': case 'b': case 'f': case 'n': case 'r':
					case 't': case 'v': case '0': case '\"': 
					case STRING_ESCAPE_SYMBOL:
						state = STR_EAT;
						break;
					case 'u':
						unicount = 4;
						state = STR_UNICODE;
						break;
					case 'U':
						unicount = 8;
						state = STR_UNICODE;
						break;
					default:
						lq_error_at(scanner->file, "Unknown escape sequence found in this string literal");
						state = STR_EAT;
						lq_array_append(buffer, "t", 1);
						goto NO_BUFFER_APPEND;
				}
				break;

			case STR_UNICODE: 
				if(--unicount == 0)
					state = STR_EAT;

				if(('0' > ch || ch > '9') && ('A' > ch || ch > 'F')) {
					lq_error_at(scanner->file, "Unexpected hex number in unicode escape sequence found");
					lq_array_append(buffer, "0", 1);
					goto NO_BUFFER_APPEND;
				}
				break;

			case STR_FINAL: 
				goto RETURN_TOKEN;
		}

		lq_array_append(buffer, p, g_unichar_to_utf8(ch, NULL));

NO_BUFFER_APPEND:
		p = g_utf8_next_char(p);
		ch = g_utf8_get_char(p);
	}

RETURN_TOKEN:
	*stream = p;

	lq_array_append(buffer, "\0", 1);

	return create_token(LQ_TOKEN_STRING, lq_array_pointer(buffer), lq_array_size(buffer), LINE(scanner));
}


static struct LqToken*
read_raw_string(struct LqContext* scanner, byte** stream)
{
	struct LqArray* buffer = scanner->buffer;
	byte* p = g_utf8_next_char(*stream);
	gunichar ch = g_utf8_get_char(p);

	while(TRUE) {
		switch(ch) {
			case '\0':
				lq_error_at(scanner->file, "Unexpected end of file found in unclosed string");
				goto RETURN_TOKEN;
			case '\\':
				if(g_utf8_get_char(p + 1) == '\'') {
					lq_array_append(buffer, p, 1);
					++p;
					break;
				}
			case '\'':
				goto RETURN_TOKEN;

			case '\n':
			case '\r':
				lq_error_at(scanner->file, "Unexpected newline/carriage return found in this raw string");
				--p;
				goto RETURN_TOKEN;
			default:
				break;
		}

		lq_array_append(buffer, p, g_unichar_to_utf8(ch, NULL));
		p = g_utf8_next_char(p);
		ch = g_utf8_get_char(p);
	}

RETURN_TOKEN:
	*stream = p;

	lq_array_append(buffer, "\0", 1);

	return create_token(LQ_TOKEN_RAWSTRING, lq_array_pointer(buffer), lq_array_size(buffer), LINE(scanner));
}


static struct LqToken*
read_character(struct LqContext* scanner, byte** stream)
{
	assert(scanner != 0);

	struct LqArray* buffer = scanner->buffer;
	byte* p = g_utf8_next_char(*stream);
	gunichar ch = g_utf8_get_char(p);

	switch(ch) {
		case '\0':
			lq_error_at(scanner->file, "Unexpected end of file found in character literal");
			goto RETURN_TOKEN;
		case '\n':
		case '\r':
			lq_error_at(scanner->file, "Unexpected newline/carriage return found in this character literal");
			lq_array_append(buffer, "0", 1);
			goto RETURN_TOKEN;
			// TODO: Escape sequences
		default:
			lq_array_append(buffer, p, g_unichar_to_utf8(ch, NULL));
			p = g_utf8_next_char(p);
	}

RETURN_TOKEN:
	*stream = p;

	lq_array_append(buffer, "\0", 1);

	return create_token(LQ_TOKEN_CHARACTER, lq_array_pointer(buffer), lq_array_size(buffer), LINE(scanner));
}



static struct LqToken*
read_token(struct LqContext* scanner)
{
	assert(scanner!= 0);

	struct LqToken* token = 0;

	struct LqArray* lines = scanner->file->lines;
	byte* p = scanner->stream;

        //TODO: this is simple wrong
        byte* line_end = (lq_array_size(lines) > 0) 
		        ? lq_cast(byte*, lq_cast(struct LqLine*, lq_array_last(lines))->begin)
		        : scanner->file->srcbegin;

	while(token == 0) {
		lq_array_clear(scanner->buffer);
		assert(lq_array_size(scanner->buffer) == 0);

		gunichar ch = g_utf8_get_char(p);

		if(g_unichar_isspace(ch) && (ch != '\n' && ch != '\r')) {
			p = g_utf8_next_char(p);
			scanner->stream = p;
			continue;
		}

		switch(ch) {
			case '\0':
				token = create_token(LQ_TOKEN_EOF, 0, 0, LINE(scanner));
				break;

			case '\r':
				if( (g_utf8_get_char(p + 1) == '\n' ) )
					p++;

			case '\n':
				token = create_token(LQ_TOKEN_NEWLINE, 0, 0, LINE(scanner) + 1);
				struct LqLine line = { p + 1, line_end - p };
				lq_array_append(lines, &line, 1);
				break;

			case '#':
				while(ch != '\n' && ch != '\r' && ch != '\0') {
					p = g_utf8_next_char(p);
					ch = g_utf8_get_char(p);
				}
				scanner->stream = p;
				continue;

			case '{':
			case '}':
			case '(':
			case ')':
			case '[':
			case ']':
			case '&':
			case ';':

			case ',':
			case '~':
			case '*':
			case '.':
			case '_':
			case '/':
			case '^':
				token = create_token(ch, 0, 0, LINE(scanner));
				break;
			case '|':
				if(g_utf8_get_char(p + 1) == '|') {
					token = create_token(LQ_TOKEN_DOUBLEPIPE, 0, 0, LINE(scanner));
					++p;
				} else {
					token = create_token(LQ_TOKEN_PIPE, 0, 0, LINE(scanner));
				}
				break;

			case '!':
				if(g_utf8_get_char(p + 1) == '!') {
					token = create_token(LQ_TOKEN_DOUBLEEXCLAMATION, 0, 0, LINE(scanner));
					++p;
				} else {
					token = create_token(LQ_TOKEN_EXCLAMATION, 0, 0, LINE(scanner));
				}
				break;
					

			case ':':
				ch = g_utf8_get_char(p + 1);

				if(ch == ':') {
					token = create_token(LQ_TOKEN_DOUBLECOLON, 0, 0, LINE(scanner));
					++p;
				} else {
					token = create_token(LQ_TOKEN_COLON, 0, 0, LINE(scanner));
				}
				break;


			case '+':
				ch = g_utf8_get_char(p + 1);

				if(ch == '>')  {
					token = create_token(LQ_TOKEN_BITSHIFTRIGHT, 0, 0, LINE(scanner));
					++p;
				}
				else if(ch == '+') {
					token = create_token(LQ_TOKEN_CONCAT, 0, 0, LINE(scanner));
					++p;
				}
				else {
					token = create_token(LQ_TOKEN_PLUS, 0, 0, LINE(scanner));
				}
				break;

			case '-':
				ch = g_utf8_get_char(p + 1);

				if(ch == '-') {
					token = create_token(LQ_TOKEN_REDUCE, 0, 0, LINE(scanner));
					++p;
				}
				else if(ch == '>') {
					token = create_token(LQ_TOKEN_ARROWRIGHT, 0, 0, LINE(scanner));
					++p;
				}
				else {
					token = create_token(LQ_TOKEN_MINUS, 0, 0, LINE(scanner));
				}
				break;


			case '<':
				ch = g_utf8_get_char(p + 1);

				if(ch == '<') {
					token = create_token(LQ_TOKEN_ARRAYLEFTBRACE, 0, 0, LINE(scanner));
					++p;
				}
				else if(ch == '-') {
					token = create_token(LQ_TOKEN_ARROWLEFT, 0, 0, LINE(scanner));
					++p;
				}
				else if(ch == '+') {
					token = create_token(LQ_TOKEN_BITSHIFTLEFT, 0, 0, LINE(scanner));
					++p;
				}
				else if(ch == '>') {
					token = create_token(LQ_TOKEN_NOTEQUAL, 0, 0, LINE(scanner));
					++p;
				}
				else {
					token = create_token(LQ_TOKEN_LESS, 0, 0, LINE(scanner));
				}
				break;

			case '>':
				ch = g_utf8_get_char(p + 1);
				++p;

				if(ch == '>') {
					token = create_token(LQ_TOKEN_ARRAYRIGHTBRACE, 0, 0, LINE(scanner));
				}
				else if(ch == '=') {
					token = create_token(LQ_TOKEN_GREATEREQUAL, 0, 0, LINE(scanner));
				} 
				else if(ch == '<') {
					token = create_token(LQ_TOKEN_BXOR, 0, 0, LINE(scanner));
				}
				else {
					token = create_token(LQ_TOKEN_GREATER, 0, 0, LINE(scanner));
					p--;
				}
				break;

			case '\\':
				token = read_character(scanner, &p);
				--p;
				break;

			case '=':
				ch = g_utf8_get_char(p + 1);
				++p;

				if(ch == '=') {
					token = create_token(LQ_TOKEN_EQUAL, 0, 0, LINE(scanner));
				} 
				else if(ch == '<') {
					token = create_token(LQ_TOKEN_LESSEQUAL, 0, 0, LINE(scanner));
				} 
				else {
					token = create_token(LQ_TOKEN_ASSIGN, 0, 0, LINE(scanner));
					--p;
				}
				break;

			case '0': case '1': case '2': case '3': case '4':
			case '5': case '6': case '7': case '8': case '9':
				token = read_number(scanner, &p);
				p--;
				break;

			case '"':
				token = read_string(scanner, &p);
				break;

			case '\'':
				token = read_raw_string(scanner, &p);
				break;

			default:
				if(g_unichar_isalpha(ch) == TRUE) {
					token = read_identifier(scanner, &p);
				} else {
					lq_error_at(scanner->file, "unexpected unicode character found");

					// number of bytes for unicode character ch
					p = g_utf8_next_char(p);
				}
				break;
		}
	}

	if(token->value != LQ_TOKEN_EOF)
		scanner->stream = ++p;

	return token;
}


struct LqToken
lq_scanner_next(struct LqContext* scanner)
{
	assert(scanner != 0);

	if(scanner->current != 0)
		free(scanner->current);

	if(lq_queue_size(scanner->peek) > 0)
		scanner->current = lq_queue_dequeue(scanner->peek);
	else
		scanner->current = read_token(scanner);

	return *(scanner->current);
}


struct LqToken
lq_scanner_peek(struct LqContext* scanner, size_t index)
{
	assert(scanner != 0);

	size_t peeksize = lq_queue_size(scanner->peek);
	int diff = (index + 1) - peeksize;

	while(diff-- > 0) {
		struct LqToken* tail = lq_queue_rear(scanner->peek);

		if(tail == 0 || tail->value != LQ_TOKEN_EOF)
			lq_queue_enqueue(scanner->peek, read_token(scanner));
		else
			diff = -1;

	}

	return *(struct LqToken*) ((lq_queue_size(scanner->peek) > index) 
			? lq_queue_nth(scanner->peek, index)
			: lq_queue_rear(scanner->peek));
}


const byte*
lq_tok_to_string(enum LqTOK token)
{
	switch(token) {
		case LQ_TOKEN_BOF:
			return "BEGIN";
		case LQ_TOKEN_EOF:	
			return "EOF";
		case LQ_TOKEN_NEWLINE:
			return "NEWLINE";
		case LQ_TOKEN_CURVEDLEFTBRACE:
			return "{";
		case LQ_TOKEN_CURVEDRIGHTBRACE:
			return "}";
		case LQ_TOKEN_ROUNDLEFTBRACE:
			return "(";
		case LQ_TOKEN_ROUNDRIGHTBRACE:
			return ")";
		case LQ_TOKEN_ARRAYLEFTBRACE:
			return "<<";
		case LQ_TOKEN_ARRAYRIGHTBRACE:
			return ">>";
		case LQ_TOKEN_SQUARELEFTBRACE:
			return "[";
		case LQ_TOKEN_SQUARERIGHTBRACE:
			return "]";
		case LQ_TOKEN_DOT:
			return ".";
		case LQ_TOKEN_BACKSLASH:
			return "\\";
		case LQ_TOKEN_COMMA:
			return ",";
		case LQ_TOKEN_COLON:
			return ":";
		case LQ_TOKEN_DOUBLECOLON:
			return "::";
		case LQ_TOKEN_UNDERSCORE:
			return "_";
		case LQ_TOKEN_PIPE:
			return "|";
		case LQ_TOKEN_DOUBLEPIPE:
			return "||";
		case LQ_TOKEN_SEMICOLON:
			return ";";
		case LQ_TOKEN_PLUS:
			return "+";
		case LQ_TOKEN_MINUS:
			return "-";
		case LQ_TOKEN_POWER:
			return "^";
		case LQ_TOKEN_DIVIDE:
			return "/";
		case LQ_TOKEN_MULTIPLY:
			return "*";
		case LQ_TOKEN_MODULO:
			return "mod";
		case LQ_TOKEN_CONCAT:
			return "++";
		case LQ_TOKEN_REDUCE:
			return "--";
		case LQ_TOKEN_ASSIGN:
			return "=";
		case LQ_TOKEN_EQUAL:
			return "==";
		case LQ_TOKEN_NOTEQUAL:
			return "<>";
		case LQ_TOKEN_GREATER:
			return ">";
		case LQ_TOKEN_GREATEREQUAL:
			return ">=";
		case LQ_TOKEN_LESS:
			return "<";
		case LQ_TOKEN_LESSEQUAL:
			return "=<";
		case LQ_TOKEN_ARROWRIGHT:
			return "->";
		case LQ_TOKEN_ARROWLEFT:
			return "<-";
		case LQ_TOKEN_AND:
			return "AND";
		case LQ_TOKEN_OR:
			return "OR";
		case LQ_TOKEN_NOT:
			return "NOT";
		case LQ_TOKEN_BAND:
			return "&";
		case LQ_TOKEN_BNOT:
			return "~";
		case LQ_TOKEN_BXOR:
			return "><";
		case LQ_TOKEN_BITSHIFTLEFT:
			return "<+";
		case LQ_TOKEN_BITSHIFTRIGHT:
			return "+>";
		case LQ_TOKEN_DOUBLEEXCLAMATION:
			return "!!";
		case LQ_TOKEN_EXCLAMATION:
			return "!";
		case LQ_TOKEN_HEXNUMBER:
			return "HEX";
		case LQ_TOKEN_DECNUMBER:
			return "DEC";
		case LQ_TOKEN_BINNUMBER:
			return "BIN";
		case LQ_TOKEN_OCTNUMBER:
			return "OCT";
		case LQ_TOKEN_FLOATPOINT:
			return "FLOATPOINT";
		case LQ_TOKEN_STRING:
			return "STRING";
		case LQ_TOKEN_RAWSTRING:
			return "RAWSTRING";
		case LQ_TOKEN_CHARACTER:
			return "CHAR";
		case LQ_TOKEN_ATOM:
			return "ATOM";
		case LQ_TOKEN_IDENTIFIER:
			return "ID";
		case LQ_TOKEN_EXTIDENTIFIER:
			return "EXTID";
		case LQ_TOKEN_IF:
			return "IF";
		case LQ_TOKEN_THEN:
			return "THEN";
		case LQ_TOKEN_ELSE:
			return "ELSE";
		case LQ_TOKEN_MATCH:
			return "MATCH";
		case LQ_TOKEN_BY:
			return "BY";
                case LQ_TOKEN_MODULE:
                        return "MODULE";
		case LQ_TOKEN_FUN:
			return "FUN";
		case LQ_TOKEN_LET:
			return "LET";
	}

	return "UNKNOWN";
}


